import urllib.request
import csv
import re
import json

#csv_file = open("rent.csv","w",encoding='GBK')
#csv_writer = csv.writer(csv_file, delimiter=',')

class Spider:
    def loadPage(self,page):
        url = "http://cd.meituan.com/meishi/b24737/pn"+str(page)+"/"

        #user-Agent头
        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"
        headers = {"User-Agent":user_agent}
        req = urllib.request.Request(url,headers = headers)
        response = urllib.request.urlopen(req)
        html =str(response.read(),'utf-8')


        #找到商家信息的内容为：{"poiId":xxx}
        #re.S 如果没有re.S,则是只匹配一行有没有符合规则的字符串，如果没有则匹配下一行重新匹配
        #如果加上re.S,则是将所有的字符串按一个整体进行匹配

        pattern = re.compile(r'{"poiId":.*?}',re.S)
        item_list = pattern.findall(html)#获取数据

        #dictinfo = json.loads(item_list[0])#把字符串转化为字典

        list = []#存放数据的数组

        for data in item_list:
            dictinfo = json.loads(data)
           # csv_writer.writerow([dictinfo["title"],dictinfo["address"],dictinfo["avgScore"],dictinfo["avgPrice"]])



if __name__ == "__main__":
    mySpider = Spider()

    for i in range(1,3):
        print("fecth:Page"+str(i))
        mySpider.loadPage(i)

    #csv_file.close()
